#include "PagePreprocess/PageLib.h"
#include "PagePreprocess/DirScanner.h"

PageLib::PageLib(DirScanner &dirs, FileProcessor &fileproc)
    : _dirScanner(dirs), _fileproc(fileproc)
{
}
// TODO: 实现创建网页库功能
void PageLib::create()
{
    //清理网页库，删除文件"data/ripepage.dat"
    remove("data/ripepage.dat");
    //清理偏移库，删除文件"data/offsetLib.dat"
    remove("data/offsetLib.dat");

    for (auto &it : _dirScanner.getfiles())
    {
        LogDebug("filename: %s", it.c_str());
        // 处理每一个xml文件
        _fileproc.process(it);
        //存入网页库
        store_ripepage(_fileproc.getWebPages());
        //清理容器，防止内存不够
        _fileproc.clearwebpages();
    }
    store_offsetLib();
}

// 实现网页库储存功能
void PageLib::store_ripepage(const vector<WebPage> &web)
{
    static int offsize = 0;
    ofstream ofs("data/ripepage.dat", std::ios::app);
   
    if (!ofs)
    {
        LogError("open data/ripepage.dat fail!");
        return;
    }

    for (size_t idx = 0; idx != web.size(); ++idx)
    {
        ostringstream oss;
        oss << "<doc>\n\t<docid>" << web[idx]._id
            << "</docid>\n\t<title>" << web[idx]._title
            << "</title>\n\t<link>" << web[idx]._link
            << "</link>\n\t<description>" << web[idx]._description << "</description>\n</doc>";
        oss << '\n';

        string str = oss.str();
        _offsetLib[web[idx]._id].first = offsize;
        _offsetLib[web[idx]._id].second = str.size();
        offsize += str.size();

        ofs << str;
      
    }

    ofs.close();
}

void PageLib::store_offsetLib()
{

    ofstream ofs("data/offsetLib.dat");
    if (!ofs)
    {
        LogError("open data/offsetLib.dat fail!");
        return;
    }
    for(auto &it : _offsetLib){
        ofs<<it.first <<" "<<it.second.first<<" "<<it.second.second<<"\n";
    }
}

